#!/usr/bin/env python
# -*- coding:utf-8 -*-

"""
aaaaaa
"""

import json
import re
import sys

import requests
from bs4 import BeautifulSoup
from scpy.logger import get_logger
import utils

logger = get_logger(__file__)

reload(sys)
sys.setdefaultencoding("utf-8")


def find_citys():
    content = utils.get_url_content('http://fangjia.fang.com/')

    soup = BeautifulSoup(content, from_encoding='gb2312')
    citys_list = soup.find_all('div', attrs={'class': 'city20141104nr'})
    city_dict = {u'北京': 'http://fangjia.fang.com/pghouse-c0bj/'}
    for citys in citys_list:
        for city in citys.find_all('a'):
            href = city['href']
            if href.startswith('/'):
                city_dict[city.getText()] = 'http://fangjia.fang.com/pghouse-c0%s/' % href.replace('/', '')
    return city_dict


def find_districts(city, url):
    content = utils.get_url_content(url).decode('gb2312', 'ignore')
    soup = BeautifulSoup(content, from_encoding='gb2312')
    dl = soup.find('dl', attrs={'id': 'dl_quxian'})
    districts = []
    for distric_url, district in re.findall(ur'<a href="(.+?)"[ ]?>(.+?)</a>', unicode(dl)):
        districts.append({
            'city': city,
            'district': district,
            'url': 'http://fangjia.fang.com%s' % distric_url
        })
    return districts


def find_shangquan(district):
    shangquan_list = []
    content = utils.get_url_content(district['url'])

    soup = BeautifulSoup(content, from_encoding='gbk')
    shangquan = soup.find('div', attrs={'id': 'div_shangquan'})
    if not shangquan:
        return shangquan_list
    for sq in shangquan.find_all('a'):
        text = utils.deal_str(sq.getText())
        if text == u'不限':
            continue
        href = sq['href']
        if href.endswith('/'):
            href = href[:-1]
        shangquan_list.append({
            '_id': '%s#%s#%s' % (district['city'], district['district'], text),
            'city': district['city'],
            'district': district['district'],
            'shangquan': text,
            'url': 'http://fangjia.fang.com%s-j360/' % href
        })
    return shangquan_list


def find_all_districts(citys):
    for index, city in enumerate(citys):
        logger.info('now city : %s(total %d, now %d).' % (city, len(citys), index))
        districts = find_districts(city, citys[city])

        for district in districts:
            logger.info('now find shangquan of district : %s%s' % (district['city'], district['district']))
            lst = find_shangquan(district)
            if not lst:
                logger.info('find error at %s.' % str(district))
            saved = utils.save_2_db(lst, 'fangjia', 'district')
            logger.info('saved shangquan %d/%d' % (saved, len(lst)))

if __name__ == '__main__':
    # find_all_districts(find_citys())
    print __doc__